import requests
from lxml import etree
from selenium import webdriver
import time

headers = {
    'User-Agent':"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36"
}

url = "https://search.jd.com/Search?keyword=python&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=python&page=3&s=57&click=0"



def requests_fetch(url):
    return requests.get(url=url,headers=headers)

def requests_jd(url):
    response = requests_fetch(url)
    html_doc = etree.HTML(response.text)
    jd_list = html_doc.xpath('//div[@id="J_goodsList"]/ul/li')
    print(f"book_count:{len(jd_list)}")

def fetch_website(url):
    driver = webdriver.Firefox()
    driver.get(url)
    return driver

def script_execute(driver,js):
    try:
        driver.execute_script(js)
        return True
    except Exception as e:
        print(f"script_execute {js} has error with {e}")
        return False

def selenium_jd(url):
    driver = fetch_website(url)
    time.sleep(1)
    js = "window.scrollTo(0, document.body.scrollHeight)"
    # js = "window.scrollTo(0, 1000)"
    # # 上面纵坐标为1000的位置上，再加上100，移动到1100的位置上
    # js = "window.scrollBy(0, 100)"

    script_execute(driver,js)
    # 这里要等底部的数据加载
    time.sleep(2)
    book_list = driver.find_element_by_class_name('gl-item')
    print(f"book_count:{len(book_list)}")
    time.sleep(10)
    driver.quit()

if __name__ == "__main__":
    selenium_jd(url)
    # requests_jd(url)



